This is a Project about skin cancer detection using AI¶
First we import all the libraries we will be using¶
In [1]:
import os
import cv2
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import torch
import torch.nn as nn
import torchvision.transforms as transforms
import torch.optim as optim
import torchvision.models as models
import time
import itertools
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import confusion_matrix
from torch.utils.data import DataLoader, Dataset
from torchvision import models
from PIL import Image
from torchvision.models import ResNet50_Weights
from sklearn.model_selection import train_test_split
from torchvision import transforms
from pytorch_grad_cam import GradCAM
from pytorch_grad_cam.utils.image import preprocess_image
from sklearn.utils.class_weight import compute_class_weight
from torch.optim.lr_scheduler import StepLR
import torch.optim.lr_scheduler as lr_scheduler
from torch.optim import Adam
We will be loading our dataset which contain about 10,000 images from ISIC(HAM10000)¶
In [2]:
# Define BASE_DIR and derived paths
BASE_DIR = r"E:\afeka\finalProject"
MEDIA_ROOT = os.path.join(BASE_DIR, 'JupyterLab')
ISIC_DIR = os.path.join(MEDIA_ROOT, 'ISIC-images')
metadata_path = os.path.join(ISIC_DIR, 'metadata.csv')
root_dir = ISIC_DIR
# Load metadata
metadata = pd.read_csv(metadata_path)
# Fix image filenames (append .jpg)
metadata["image_name"] = metadata["isic_id"] + ".jpg"
# Check if images exist
missing_files = [f for f in metadata["image_name"] if not os.path.exists(os.path.join(root_dir, f))]
if missing_files:
print("Missing files:", missing_files[:5])
else:
print("All images exist.")
All images exist.
In [3]:
print(metadata.head())
isic_id attribution \
0 ISIC_0024306 ViDIR Group, Department of Dermatology, Medica...
1 ISIC_0024307 ViDIR Group, Department of Dermatology, Medica...
2 ISIC_0024308 ViDIR Group, Department of Dermatology, Medica...
3 ISIC_0024309 ViDIR Group, Department of Dermatology, Medica...
4 ISIC_0024310 ViDIR Group, Department of Dermatology, Medica...
copyright_license age_approx anatom_site_general anatom_site_special \
0 CC-BY-NC 45.0 NaN NaN
1 CC-BY-NC 50.0 lower extremity NaN
2 CC-BY-NC 55.0 NaN NaN
3 CC-BY-NC 40.0 NaN NaN
4 CC-BY-NC 60.0 anterior torso NaN
benign_malignant concomitant_biopsy diagnosis diagnosis_1 \
0 benign False nevus Benign
1 benign False nevus Benign
2 benign False nevus Benign
3 benign False nevus Benign
4 malignant True melanoma Malignant
diagnosis_2 diagnosis_3 \
0 Benign melanocytic proliferations Nevus
1 Benign melanocytic proliferations Nevus
2 Benign melanocytic proliferations Nevus
3 Benign melanocytic proliferations Nevus
4 Malignant melanocytic proliferations (Melanoma) Melanoma, NOS
diagnosis_confirm_type image_type lesion_id melanocytic \
0 serial imaging showing no change dermoscopic IL_7252831 True
1 serial imaging showing no change dermoscopic IL_6125741 True
2 serial imaging showing no change dermoscopic IL_3692653 True
3 serial imaging showing no change dermoscopic IL_0959663 True
4 histopathology dermoscopic IL_8194852 True
sex image_name
0 male ISIC_0024306.jpg
1 male ISIC_0024307.jpg
2 female ISIC_0024308.jpg
3 male ISIC_0024309.jpg
4 male ISIC_0024310.jpg
now we will print all the labels and the count for the label "diagnosis"¶
In [4]:
print(metadata.columns.tolist()) # List all column names
print(metadata["diagnosis"].value_counts()) # Check label distribution
['isic_id', 'attribution', 'copyright_license', 'age_approx', 'anatom_site_general', 'anatom_site_special', 'benign_malignant', 'concomitant_biopsy', 'diagnosis', 'diagnosis_1', 'diagnosis_2', 'diagnosis_3', 'diagnosis_confirm_type', 'image_type', 'lesion_id', 'melanocytic', 'sex', 'image_name'] diagnosis nevus 7737 pigmented benign keratosis 1338 melanoma 1305 basal cell carcinoma 622 squamous cell carcinoma 229 vascular lesion 180 dermatofibroma 160 actinic keratosis 149 Name: count, dtype: int64
Visualize how common each skin condition is in the dataset using a horizontal bar chart and pie chart¶
In [5]:
# Count diagnoses
diagnosis_counts = metadata['diagnosis'].value_counts().reset_index()
diagnosis_counts.columns = ['diagnosis', 'count']
# Plot
plt.figure(figsize=(10, 6))
sns.barplot(
data=diagnosis_counts,
x='count',
y='diagnosis',
hue='diagnosis', # This applies the palette per category
dodge=False,
palette="viridis",
legend=False
)
plt.title("Distribution of Diagnoses")
plt.xlabel("Count")
plt.ylabel("Diagnosis")
plt.tight_layout()
plt.show()
In [6]:
# Get all diagnosis counts
diagnosis_counts = metadata['diagnosis'].value_counts()
# Plot
plt.figure(figsize=(10, 10))
plt.pie(
diagnosis_counts.values,
labels=diagnosis_counts.index,
autopct='%1.1f%%',
startangle=140
)
plt.title("Diagnosis Distribution ")
plt.axis('equal')
plt.show()
Mapping Lesions to Numeric Labels¶
In [7]:
# Identify the rows where 'diagnosis_1' is 'Indeterminate'
rows_to_drop = metadata[metadata['diagnosis_1'] == 'Indeterminate'].index
# Drop the rows from the dataframe
metadata.drop(rows_to_drop, inplace=True)
#reset the index
metadata.reset_index(drop=True, inplace=True)
In [8]:
# Define the label mapping function
def map_labels(row):
# Nevus mapping
if row["diagnosis"] == "nevus":
return 0 # Nevus
# Melanoma mapping
elif row["diagnosis"] == "melanoma":
return 1 # Melanoma
# If diagnosis is from other specific types and diagnosis_1 is Benign, classify as Nevus
elif (row["diagnosis"] in ["pigmented benign keratosis", "basal cell carcinoma", "squamous cell carcinoma",
"vascular lesion", "dermatofibroma", "actinic keratosis"]) and row["diagnosis_1"] == "Benign":
return 0 # Nevus
# For Other lesion (Malignant)
elif row["diagnosis"] == "Other lesion":
if row["diagnosis_1"] in ["Malignant"]:
return 2 # Other lesion (Malignant)
# If diagnosis is one of the other types and diagnosis_1 is Malignant, classify as Other lesion
if row["diagnosis_1"] in ["Malignant"]:
return 2 # Other lesion
return 2 # Default to Other lesion for any other case
# Apply the mapping function to create a new column 'mapped_label'
metadata["mapped_label"] = metadata.apply(map_labels, axis=1)
# Print class distribution
class_distribution = metadata["mapped_label"].value_counts().sort_index()
class_labels = {
0: "Nevus",
1: "Melanoma",
2: "Other lesion"
}
# Output the class distribution
print("Class Distribution:")
for label, count in class_distribution.items():
print(f"{class_labels[label]}: {count}")
Class Distribution: Nevus: 9415 Melanoma: 1305 Other lesion: 851
Creating and Loading the ISICDataset for Skin Lesion Classification¶
In [9]:
class ISICDataset(Dataset):
def __init__(self, metadata_path, root_dir, transform=None):
"""
Args:
metadata_path (str): Path to the CSV file containing metadata.
root_dir (str): Directory with all the images.
transform (callable, optional): Transformations to be applied to images.
"""
self.metadata = pd.read_csv(metadata_path)
self.metadata["image_name"] = self.metadata["isic_id"] + ".jpg" # Fix filenames
self.root_dir = root_dir
self.transform = transform
# Apply the label mapping logic and add the 'mapped_label' column
self.metadata["mapped_label"] = self.metadata.apply(map_labels, axis=1)
def __len__(self):
return len(self.metadata)
def __getitem__(self, idx):
img_name = self.metadata.iloc[idx]["image_name"]
label = self.metadata.iloc[idx]["mapped_label"] # Use 'mapped_label' instead of the old 'label'
img_path = os.path.join(self.root_dir, img_name)
# Open image
image = Image.open(img_path).convert("RGB") # Ensure 3 channels
# Apply transformations
if self.transform:
image = self.transform(image)
return image, label
# Define transformations (data augmentation)
transform = transforms.Compose([
transforms.RandomHorizontalFlip(p=0.5), # Flip 50% of images horizontally
transforms.RandomRotation(degrees=15), # Rotate images randomly by ±15 degrees
transforms.RandomResizedCrop(224, scale=(0.8, 1.0)), # Random crop + resize
transforms.RandomAffine(degrees=0, scale=(0.9, 1.1)), # Randomly zoom image, scale range between 90% to 110%
transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1), # Adjust colors
transforms.ToTensor(), # Convert image to tensor
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) # Normalize with ImageNet mean/std
])
dataset = ISICDataset(metadata_path, root_dir, transform=transform)
dataloader = DataLoader(dataset, batch_size=32, shuffle=True)
# Test if the dataloader works
for images, labels in dataloader:
print(f"Batch of images: {images.shape}, Labels: {labels.shape}")
break # Check one batch
Batch of images: torch.Size([32, 3, 224, 224]), Labels: torch.Size([32])
Visualizing the Distribution of Lesion Types¶
In [10]:
# Define the label mapping
label_mapping = {0: "nevus", 1: "melanoma", 2: "other lesion"}
# Map the numeric labels to the string labels for better visualization
mapped_label_counts = dataset.metadata["mapped_label"].map(label_mapping).value_counts()
# Create the bar chart
plt.figure(figsize=(6, 4))
bars = plt.bar(mapped_label_counts.index, mapped_label_counts.values, color=['blue', 'gray', 'red'])
# Add count labels on top of each bar
for bar in bars:
height = bar.get_height()
plt.text(bar.get_x() + bar.get_width() / 2, height, f'{height}', ha='center', va='bottom', fontsize=10)
# Add labels and title
plt.xlabel("Lesion Type")
plt.ylabel("Count")
plt.title("Distribution of Lesion Types")
plt.xticks(ticks=range(len(mapped_label_counts.index)), labels=mapped_label_counts.index, rotation=45)
plt.show()
# Pie chart
plt.figure(figsize=(6, 6))
plt.pie(mapped_label_counts.values, labels=mapped_label_counts.index, autopct='%1.1f%%', colors=['blue', 'gray', 'red'])
plt.title("Proportion of Lesion Types")
plt.show()
Data Preprocessing: Removing Unnecessary Columns and Handling Missing Values¶
In [11]:
columns_to_remove = ['attribution', 'copyright_license', 'image_type']
metadata = metadata.drop(columns=columns_to_remove)
# Display the first few rows to confirm
print(metadata.head())
isic_id age_approx anatom_site_general anatom_site_special \
0 ISIC_0024306 45.0 NaN NaN
1 ISIC_0024307 50.0 lower extremity NaN
2 ISIC_0024308 55.0 NaN NaN
3 ISIC_0024309 40.0 NaN NaN
4 ISIC_0024310 60.0 anterior torso NaN
benign_malignant concomitant_biopsy diagnosis diagnosis_1 \
0 benign False nevus Benign
1 benign False nevus Benign
2 benign False nevus Benign
3 benign False nevus Benign
4 malignant True melanoma Malignant
diagnosis_2 diagnosis_3 \
0 Benign melanocytic proliferations Nevus
1 Benign melanocytic proliferations Nevus
2 Benign melanocytic proliferations Nevus
3 Benign melanocytic proliferations Nevus
4 Malignant melanocytic proliferations (Melanoma) Melanoma, NOS
diagnosis_confirm_type lesion_id melanocytic sex \
0 serial imaging showing no change IL_7252831 True male
1 serial imaging showing no change IL_6125741 True male
2 serial imaging showing no change IL_3692653 True female
3 serial imaging showing no change IL_0959663 True male
4 histopathology IL_8194852 True male
image_name mapped_label
0 ISIC_0024306.jpg 0
1 ISIC_0024307.jpg 0
2 ISIC_0024308.jpg 0
3 ISIC_0024309.jpg 0
4 ISIC_0024310.jpg 1
In [12]:
# Check if any missing values remain
print(metadata.isnull().sum())
isic_id 0 age_approx 380 anatom_site_general 2158 anatom_site_special 11040 benign_malignant 2529 concomitant_biopsy 0 diagnosis 0 diagnosis_1 0 diagnosis_2 0 diagnosis_3 180 diagnosis_confirm_type 0 lesion_id 0 melanocytic 0 sex 340 image_name 0 mapped_label 0 dtype: int64
In [13]:
# Fill missing values in categorical columns with 'Unknown'
metadata['anatom_site_general'] = metadata['anatom_site_general'].fillna('Unknown')
metadata['anatom_site_special'] = metadata['anatom_site_special'].fillna('Unknown')
metadata['benign_malignant'] = metadata['benign_malignant'].fillna('Unknown')
# Fill missing values in numerical columns like 'age_approx' with the median
metadata['age_approx'] = metadata['age_approx'].fillna(metadata['age_approx'].median())
# Fill missing values in categorical columns with 'Unknown' or the mode
metadata['diagnosis_3'] = metadata['diagnosis_3'].fillna('Unknown')
metadata['sex'] = metadata['sex'].fillna(metadata['sex'].mode()[0])
In [14]:
print(metadata.isnull().sum()) # Check if any columns still have missing values
isic_id 0 age_approx 0 anatom_site_general 0 anatom_site_special 0 benign_malignant 0 concomitant_biopsy 0 diagnosis 0 diagnosis_1 0 diagnosis_2 0 diagnosis_3 0 diagnosis_confirm_type 0 lesion_id 0 melanocytic 0 sex 0 image_name 0 mapped_label 0 dtype: int64
In [15]:
print(metadata.head()) # Check the first few rows of the DataFrame
isic_id age_approx anatom_site_general anatom_site_special \
0 ISIC_0024306 45.0 Unknown Unknown
1 ISIC_0024307 50.0 lower extremity Unknown
2 ISIC_0024308 55.0 Unknown Unknown
3 ISIC_0024309 40.0 Unknown Unknown
4 ISIC_0024310 60.0 anterior torso Unknown
benign_malignant concomitant_biopsy diagnosis diagnosis_1 \
0 benign False nevus Benign
1 benign False nevus Benign
2 benign False nevus Benign
3 benign False nevus Benign
4 malignant True melanoma Malignant
diagnosis_2 diagnosis_3 \
0 Benign melanocytic proliferations Nevus
1 Benign melanocytic proliferations Nevus
2 Benign melanocytic proliferations Nevus
3 Benign melanocytic proliferations Nevus
4 Malignant melanocytic proliferations (Melanoma) Melanoma, NOS
diagnosis_confirm_type lesion_id melanocytic sex \
0 serial imaging showing no change IL_7252831 True male
1 serial imaging showing no change IL_6125741 True male
2 serial imaging showing no change IL_3692653 True female
3 serial imaging showing no change IL_0959663 True male
4 histopathology IL_8194852 True male
image_name mapped_label
0 ISIC_0024306.jpg 0
1 ISIC_0024307.jpg 0
2 ISIC_0024308.jpg 0
3 ISIC_0024309.jpg 0
4 ISIC_0024310.jpg 1
In [16]:
label_encoder = LabelEncoder()
metadata['sex'] = label_encoder.fit_transform(metadata['sex'])
CustomImageDataset: A PyTorch Dataset for Loading and Transforming Images¶
In [17]:
class CustomImageDataset(Dataset):
def __init__(self, metadata_df, root_dir, transform=None):
"""
Args:
metadata_df (DataFrame): DataFrame containing image metadata.
root_dir (str): Directory with all the images.
transform (callable, optional): Optional transform to be applied on a sample.
"""
self.metadata = metadata_df # The DataFrame containing metadata
self.metadata["mapped_label"] = self.metadata.apply(map_labels, axis=1) # Map diagnosis to numeric labels (axis=1 applies to rows)
self.root_dir = root_dir
self.transform = transform
def __len__(self):
# Return the total number of images in the dataset
return len(self.metadata)
def __getitem__(self, idx):
# Get image name (ensure you append the file extension like '.jpg')
img_name = self.metadata.iloc[idx, 0] + ".jpg" # Assuming the first column has image names without extension
img_path = os.path.join(self.root_dir, img_name) # Get full image path
# Check if the file exists (optional but good practice)
if not os.path.exists(img_path):
raise FileNotFoundError(f"Image {img_name} not found in {self.root_dir}")
# Load the image
image = Image.open(img_path).convert("RGB") # Open and convert to RGB if not already
# Get the corresponding label (mapped label from the DataFrame)
label = self.metadata.iloc[idx]["mapped_label"]
# Apply transformations if provided (e.g., resize, normalize, etc.)
if self.transform:
image = self.transform(image)
return image, label # Return the image and its label
run the model on the device CPU/GPU¶
In [18]:
print(torch.cuda.is_available())
print(torch.cuda.get_device_name(0))
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
True NVIDIA GeForce RTX 4080 SUPER
Setting Up ResNet-50 with Class Weights and Optimization¶
In [19]:
# feature_extract is a boolean that defines if we are finetuning or feature extracting.
# If feature_extract = False, the model is finetuned and all model parameters are updated.
# If feature_extract = True, only the last layer parameters are updated, the others remain fixed.
def set_parameter_requires_grad(model, feature_extracting):
if feature_extracting:
for param in model.parameters():
param.requires_grad = False
In [20]:
class_weights = compute_class_weight(
class_weight="balanced", # Automatically balance class weights
classes=np.array([0, 1, 2]), # Specify class labels explicitly
y=metadata["mapped_label"].values # Convert the 'mapped_label' column to a numpy array
)
class_weights = torch.tensor(class_weights, dtype=torch.float32).to(device)
print("Class Weights:", class_weights)
criterion = torch.nn.CrossEntropyLoss(weight=class_weights) # Loss function for multi-class classification
# Load ResNet-50 model with updated weights parameter
model = models.resnet50(weights=ResNet50_Weights.DEFAULT)
set_parameter_requires_grad(model, feature_extracting=False)
# Modify the final fully connected layer
num_ftrs = model.fc.in_features
model.fc = nn.Sequential(
nn.Linear(num_ftrs, 512), # Fully connected layer with 512 units
nn.ReLU(), # Apply ReLU activation
nn.Dropout(0.5), # Apply 50% dropout
nn.Linear(512, 3) # Final output layer with 3 classes ('nevus', 'melanoma', 'other lesion')
)
# Initialize optimizer with L2 regularization (weight decay)
# optimizer = torch.optim.SGD(model.parameters(), lr=0.003, momentum=0.9, weight_decay=1e-4) # L2 regularization through weight_decay
# Adam can be used instead If needed:
optimizer = Adam(
model.parameters(),
lr=0.0001,
betas=(0.9, 0.999),
eps=1e-7,
amsgrad=False
)
Class Weights: tensor([0.4097, 2.9556, 4.5323], device='cuda:0')
In [21]:
# Split the data into training and validation sets (80% training, 20% validation)
train_metadata, val_metadata = train_test_split(metadata, test_size=0.2, random_state=42)
# Check the split sizes
print(f"Training set size: {len(train_metadata)}")
print(f"Validation set size: {len(val_metadata)}")
Training set size: 9256 Validation set size: 2315
In [22]:
# Initialize the CustomImageDataset with the DataFrame
train_dataset = CustomImageDataset(metadata_df=train_metadata, root_dir=root_dir, transform=transform)
val_dataset = CustomImageDataset(metadata_df=val_metadata, root_dir=root_dir, transform=transform)
In [23]:
# Function to apply Grad-CAM
class GradCAM:
def __init__(self, model, target_layer):
self.model = model
self.target_layer = target_layer
self.gradients = None
self.activation = None
# Hook to get gradients
self.target_layer.register_forward_hook(self.save_activation)
self.target_layer.register_backward_hook(self.save_gradient)
def save_activation(self, module, input, output):
self.activation = output
def save_gradient(self, module, grad_input, grad_output):
self.gradients = grad_output[0]
def generate_cam(self, class_idx):
grad_val = self.gradients.cpu().data.numpy()
act_val = self.activation.cpu().data.numpy()
weights = np.mean(grad_val, axis=(2, 3), keepdims=True)
cam = np.sum(weights * act_val, axis=1)
# Normalize and resize CAM
cam = np.maximum(cam, 0) # ReLU
cam = cam[0] # Get first item
cam = cv2.resize(cam, (224, 224)) # Resize to match image
# Normalize CAM
cam = cam - np.min(cam)
cam = cam / np.max(cam)
return cam
Training the Model with Early Stopping and Validation¶
In [24]:
# Define a function for the training loop
def train_one_epoch(model, train_loader, criterion, optimizer, device, epoch, clip_grad_norm=None):
model.train()
running_loss = 0.0
correct_predictions = 0
total_predictions = 0
start_time = time.time() # Track epoch time
for i, (inputs, labels) in enumerate(train_loader):
inputs, labels = inputs.to(device), labels.to(device)
optimizer.zero_grad()
outputs = model(inputs)
loss = criterion(outputs, labels)
loss.backward()
# Gradient clipping (if applicable)
if clip_grad_norm:
torch.nn.utils.clip_grad_norm_(model.parameters(), clip_grad_norm)
optimizer.step()
running_loss += loss.item()
_, predicted = torch.max(outputs, 1)
correct_predictions += (predicted == labels).sum().item()
total_predictions += labels.size(0)
# Log learning rate every 100 iterations
if (i + 1) % 100 == 0:
print(f"Epoch {epoch+1}, Iteration {i + 1}/{len(train_loader)}")
# Calculate average loss and accuracy
train_loss = running_loss / len(train_loader)
train_accuracy = 100 * correct_predictions / total_predictions
# Print epoch time
epoch_time = time.time() - start_time
print(f"Epoch {epoch+1} completed in {epoch_time:.2f} seconds")
return train_loss, train_accuracy
# Define a function for the validation loop
def validate_model(model, val_loader, criterion, device, num_examples=5):
model.eval()
val_loss = 0.0
val_correct_predictions = 0
val_total_predictions = 0
# Store misclassified images, true labels, and predicted labels
misclassified_images = []
true_labels = []
predicted_labels = []
with torch.no_grad():
for inputs, labels in val_loader:
inputs, labels = inputs.to(device), labels.to(device)
outputs = model(inputs)
loss = criterion(outputs, labels)
val_loss += loss.item()
_, predicted = torch.max(outputs, 1)
val_correct_predictions += (predicted == labels).sum().item()
val_total_predictions += labels.size(0)
# Find misclassified images
misclassified_mask = predicted != labels
misclassified_images.extend(inputs[misclassified_mask].cpu())
true_labels.extend(labels[misclassified_mask].cpu())
predicted_labels.extend(predicted[misclassified_mask].cpu())
val_loss /= len(val_loader)
val_accuracy = 100 * val_correct_predictions / val_total_predictions
return val_loss, val_accuracy, misclassified_images, true_labels, predicted_labels
In [25]:
# Initialize lists to store loss and accuracy
train_losses = []
val_losses = []
train_accuracies = []
val_accuracies = []
# Set up the DataLoader
batch_size = 32
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)
# Set early stopping parameters
patience = 6 # Number of epochs to wait for improvement
best_val_loss = float('inf')
best_Accuracy = float('inf')
epochs_without_improvement = 0
num_epochs = 20 # Set the number of epochs
# Set up device
model.to(device)
# Store misclassified images at the end
final_misclassified_images = []
final_true_labels = []
final_predicted_labels = []
# Training and validation loop
for epoch in range(num_epochs):
train_loss, train_accuracy = train_one_epoch(model, train_loader, criterion, optimizer, device, epoch)
train_losses.append(train_loss)
train_accuracies.append(train_accuracy)
print(f"Epoch [{epoch+1}/{num_epochs}], Loss: {train_loss:.4f}, Accuracy: {train_accuracy:.2f}%")
val_loss, val_accuracy, misclassified_images, true_labels, predicted_labels = validate_model(model, val_loader, criterion, device)
val_losses.append(val_loss)
val_accuracies.append(val_accuracy)
# Save the last batch of misclassified images
final_misclassified_images = misclassified_images
final_true_labels = true_labels
final_predicted_labels = predicted_labels
print(f"Validation Loss: {val_loss:.4f}, Validation Accuracy: {val_accuracy:.2f}%")
# Early stopping check
if val_loss < best_val_loss:
best_val_loss = val_loss
best_accuracy = val_accuracy
epochs_without_improvement = 0
torch.save(model.state_dict(), 'resnet50_custom_best.pth')
torch.save(optimizer.state_dict(), 'resnet50_optimizer_best.pth')
print("Best Model saved")
else:
epochs_without_improvement += 1
if epochs_without_improvement >= patience:
print("Early stopping triggered!")
break
# Print final results
print(f"Finished Training and Validation, Best Model Accuracy: {best_accuracy:.2f}%, Loss: {best_val_loss:.4f}")
Epoch 1, Iteration 100/290 Epoch 1, Iteration 200/290 Epoch 1 completed in 72.73 seconds Epoch [1/20], Loss: 0.6940, Accuracy: 69.58% Validation Loss: 0.6664, Validation Accuracy: 72.40% Best Model saved Epoch 2, Iteration 100/290 Epoch 2, Iteration 200/290 Epoch 2 completed in 71.93 seconds Epoch [2/20], Loss: 0.5269, Accuracy: 74.68% Validation Loss: 0.4821, Validation Accuracy: 76.33% Best Model saved Epoch 3, Iteration 100/290 Epoch 3, Iteration 200/290 Epoch 3 completed in 71.65 seconds Epoch [3/20], Loss: 0.4498, Accuracy: 77.28% Validation Loss: 0.5578, Validation Accuracy: 84.41% Epoch 4, Iteration 100/290 Epoch 4, Iteration 200/290 Epoch 4 completed in 72.07 seconds Epoch [4/20], Loss: 0.4057, Accuracy: 79.62% Validation Loss: 0.4737, Validation Accuracy: 73.78% Best Model saved Epoch 5, Iteration 100/290 Epoch 5, Iteration 200/290 Epoch 5 completed in 71.63 seconds Epoch [5/20], Loss: 0.3706, Accuracy: 81.30% Validation Loss: 0.5036, Validation Accuracy: 85.23% Epoch 6, Iteration 100/290 Epoch 6, Iteration 200/290 Epoch 6 completed in 71.93 seconds Epoch [6/20], Loss: 0.3454, Accuracy: 82.49% Validation Loss: 0.5386, Validation Accuracy: 83.28% Epoch 7, Iteration 100/290 Epoch 7, Iteration 200/290 Epoch 7 completed in 71.80 seconds Epoch [7/20], Loss: 0.2949, Accuracy: 84.78% Validation Loss: 0.4169, Validation Accuracy: 80.69% Best Model saved Epoch 8, Iteration 100/290 Epoch 8, Iteration 200/290 Epoch 8 completed in 72.00 seconds Epoch [8/20], Loss: 0.2725, Accuracy: 86.06% Validation Loss: 0.4407, Validation Accuracy: 79.65% Epoch 9, Iteration 100/290 Epoch 9, Iteration 200/290 Epoch 9 completed in 71.53 seconds Epoch [9/20], Loss: 0.2494, Accuracy: 86.73% Validation Loss: 0.4718, Validation Accuracy: 84.79% Epoch 10, Iteration 100/290 Epoch 10, Iteration 200/290 Epoch 10 completed in 71.83 seconds Epoch [10/20], Loss: 0.2502, Accuracy: 87.06% Validation Loss: 0.5192, Validation Accuracy: 86.09% Epoch 11, Iteration 100/290 Epoch 11, Iteration 200/290 Epoch 11 completed in 71.84 seconds Epoch [11/20], Loss: 0.2191, Accuracy: 88.81% Validation Loss: 0.4157, Validation Accuracy: 80.52% Best Model saved Epoch 12, Iteration 100/290 Epoch 12, Iteration 200/290 Epoch 12 completed in 71.73 seconds Epoch [12/20], Loss: 0.2021, Accuracy: 88.95% Validation Loss: 0.5886, Validation Accuracy: 87.04% Epoch 13, Iteration 100/290 Epoch 13, Iteration 200/290 Epoch 13 completed in 71.73 seconds Epoch [13/20], Loss: 0.2071, Accuracy: 89.82% Validation Loss: 0.4686, Validation Accuracy: 85.14% Epoch 14, Iteration 100/290 Epoch 14, Iteration 200/290 Epoch 14 completed in 71.93 seconds Epoch [14/20], Loss: 0.1654, Accuracy: 91.40% Validation Loss: 0.6447, Validation Accuracy: 85.70% Epoch 15, Iteration 100/290 Epoch 15, Iteration 200/290 Epoch 15 completed in 71.50 seconds Epoch [15/20], Loss: 0.1775, Accuracy: 90.91% Validation Loss: 0.4318, Validation Accuracy: 85.83% Epoch 16, Iteration 100/290 Epoch 16, Iteration 200/290 Epoch 16 completed in 71.68 seconds Epoch [16/20], Loss: 0.1525, Accuracy: 92.34% Validation Loss: 0.4751, Validation Accuracy: 84.79% Epoch 17, Iteration 100/290 Epoch 17, Iteration 200/290 Epoch 17 completed in 71.65 seconds Epoch [17/20], Loss: 0.1455, Accuracy: 92.25% Validation Loss: 0.5289, Validation Accuracy: 88.94% Early stopping triggered! Finished Training and Validation, Best Model Accuracy: 80.52%, Loss: 0.4157
Visualizing Model Performance: Loss & Accuracy Trends¶
In [26]:
# Plot Training & Validation Loss
plt.figure(figsize=(10, 5))
plt.plot(range(1, len(train_losses) + 1), train_losses, label="Training Loss", marker='o')
plt.plot(range(1, len(val_losses) + 1), val_losses, label="Validation Loss", marker='o')
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.title("Training & Validation Loss")
plt.legend()
plt.grid()
plt.show()
# Plot Training & Validation Accuracy
plt.figure(figsize=(10, 5))
plt.plot(range(1, len(train_accuracies) + 1), train_accuracies, label="Training Accuracy", marker='o')
plt.plot(range(1, len(val_accuracies) + 1), val_accuracies, label="Validation Accuracy", marker='o')
plt.xlabel("Epochs")
plt.ylabel("Accuracy (%)")
plt.title("Training & Validation Accuracy")
plt.legend()
plt.grid()
plt.show()
Confusion Matrix and Misclassified Images Analysis¶
In [27]:
# Define a mapping for label names
label_mapping = {0: "nevus", 1: "melanoma", 2: "other lesion"}
model.eval()
y_true = []
y_pred = []
with torch.no_grad():
for images, labels in val_loader:
images = images.to(device)
labels = labels.to(device)
outputs = model(images)
predictions = outputs.max(1)[1] # Get predicted class index
y_true.extend(labels.cpu().numpy())
y_pred.extend(predictions.cpu().numpy())
# Compute confusion matrix
confusion_mtx = confusion_matrix(y_true, y_pred)
# Function to plot confusion matrix with COUNT values
def plot_confusion_matrix(cm, classes, normalize=False, title="Confusion Matrix", cmap=plt.cm.Blues):
plt.figure(figsize=(8, 6))
plt.imshow(cm, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)
if normalize:
cm = cm.astype("float") / cm.sum(axis=1)[:, np.newaxis] # Normalize by row sum
# Define threshold for color contrast
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
color = "green" if i == j else "red" # Green for correct, Red for incorrect
plt.text(j, i, f"{cm[i, j]:.2f}" if normalize else f"{cm[i, j]}",
horizontalalignment="center",
color=color, fontsize=12, fontweight="bold")
plt.ylabel("True label")
plt.xlabel("Predicted label")
plt.tight_layout()
plt.show()
# Plot confusion matrix as COUNT instead of percentage
plot_confusion_matrix(confusion_mtx, list(label_mapping.values()), normalize=False, title="Confusion Matrix")
if len(final_misclassified_images) > 0:
print(f"Showing {min(5, len(final_misclassified_images))} misclassified images from final validation:")
for i in range(min(5, len(final_misclassified_images))):
image = final_misclassified_images[i]
true_label = label_mapping[final_true_labels[i].item()]
predicted_label = label_mapping[final_predicted_labels[i].item()]
# Convert tensor to numpy and denormalize
image = image.permute(1, 2, 0).numpy()
if image.min() < 0 or image.max() > 1:
image = (image - image.min()) / (image.max() - image.min())
# Plot the image
plt.figure(figsize=(5, 5))
plt.imshow(image)
plt.title(f"True: {true_label}, Predicted: {predicted_label}")
plt.axis("off")
plt.show()
Showing 5 misclassified images from final validation:
Our model's main misclassifications occur when melanoma is predicted as nevus and the other way around when nevus is predicted as melanoma.
Nonetheless, we achieved an 80.52% prediction accuracy and a validation loss of 0.4157, which is quite decent.
Grad-CAM Visualization for Model Predictions¶
In [28]:
# Path to the directory containing images
image_dir = os.path.join(MEDIA_ROOT, 'images_ToTest', 'images')
# Get all image files in the directory
image_files = [f for f in os.listdir(image_dir) if f.endswith(('.png', '.jpg', '.jpeg'))]
# Set model to evaluation mode
model.eval()
# Define confidence threshold (adjust as needed)
confidence_threshold = 70 # Threshold in percentage
# Class labels
class_labels = ['nevus', 'melanoma', 'other lesion']
# Choose the layer for ResNet (last conv layer in layer4)
target_layer = model.layer4[-1]
# Initialize Grad-CAM
grad_cam = GradCAM(model=model, target_layer=target_layer)
# Iterate through all images
for image_file in image_files:
image_path = os.path.join(image_dir, image_file)
# Load and preprocess the image
image = Image.open(image_path)
input_tensor = transform(image).unsqueeze(0).to(device) # Add batch dimension and move to device
input_tensor.requires_grad_(True) # Enable gradient computation
# Forward pass
output = model(input_tensor)
probabilities = torch.nn.functional.softmax(output, dim=1) # Get softmax probabilities
confidence, predicted = torch.max(probabilities, 1) # Get max probability and index
# Perform backward pass to compute gradients
model.zero_grad() # Clear existing gradients
output[0, predicted.item()].backward() # Backward pass for the predicted class
predicted_class = class_labels[predicted.item()]
confidence_percentage = confidence.item() * 100 # Convert to percentage
# Extract image ID from filename
image_id = os.path.splitext(image_file)[0]
# Generate the Grad-CAM heatmap
cam = grad_cam.generate_cam(predicted.item()) # Pass the predicted class index
# Convert original image to NumPy array
image_np = np.array(image.resize((224, 224)))
# Squeeze the CAM to ensure it's 2D
cam = np.squeeze(cam)
# Normalize and convert cam to uint8
cam = cam - np.min(cam) # Shift values to positive range
cam = cam / np.max(cam) # Normalize to [0,1]
cam = np.uint8(255 * cam) # Scale to [0,255] and convert to uint8
# Overlay heatmap on image
heatmap = cv2.applyColorMap(cam, cv2.COLORMAP_JET)
superimposed_img = cv2.addWeighted(image_np, 0.6, heatmap, 0.4, 0)
# Print results with percentage confidence and display Grad-CAM
if confidence_percentage < confidence_threshold:
print(f"Image ID: {image_id}, Prediction Uncertain (Confidence: {confidence_percentage:.2f}%)")
else:
print(f"Image ID: {image_id}, Predicted Outcome: {predicted_class} (Confidence: {confidence_percentage:.2f}%)")
# Display the original and Grad-CAM result
plt.figure(figsize=(10, 5))
plt.subplot(1, 2, 1)
plt.imshow(image_np)
plt.title(f"Original Image: {image_id}")
plt.subplot(1, 2, 2)
plt.imshow(superimposed_img)
plt.title("Grad-CAM Heatmap")
plt.show()
C:\Users\mantr\AppData\Local\Programs\Python\Python312\Lib\site-packages\torch\nn\modules\module.py:1830: FutureWarning: Using a non-full backward hook when the forward contains multiple autograd Nodes is deprecated and will be removed in future versions. This hook will be missing some grad_input. Please use register_full_backward_hook to get the documented behavior. self._maybe_warn_non_full_backward_hook(args, result, grad_fn)
Image ID: Car_Image, Predicted Outcome: nevus (Confidence: 89.98%)
Image ID: ISIC_1871644, Predicted Outcome: nevus (Confidence: 99.96%)
Image ID: ISIC_2175344, Predicted Outcome: nevus (Confidence: 99.05%)
Image ID: ISIC_2310644, Predicted Outcome: melanoma (Confidence: 71.87%)
Image ID: ISIC_5178159, Predicted Outcome: melanoma (Confidence: 99.46%)
Image ID: ISIC_5562564, Predicted Outcome: nevus (Confidence: 97.37%)
Image ID: ISIC_6377613, Predicted Outcome: other lesion (Confidence: 99.39%)
Image ID: ISIC_6821316, Predicted Outcome: melanoma (Confidence: 99.28%)
Image ID: ISIC_7522225, Predicted Outcome: nevus (Confidence: 99.96%)
Image ID: ISIC_9105056, Predicted Outcome: other lesion (Confidence: 71.73%)
Image ID: ISIC_9911782, Predicted Outcome: nevus (Confidence: 99.91%)
Image ID: ISIC_9957003, Prediction Uncertain (Confidence: 62.46%)
In [ ]: